import scrapy
from tenxun1.items import Tenxun1Item
import json
import time

class TenxunSpider(scrapy.Spider):
    name = 'tenxun'
    # allowed_domains = ['https://careers.tencent.com/']
    # start_urls = ['https://careers.tencent.com/tencentcareer/api/post/Query?timestamp={}&countryId=&cityId=2&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword=python&pageIndex=1&pageSize=10&language=zh-cn&area=cn'.format(int(time.time()*1000))]
    p=1
    def start_requests(self):
        url='https://careers.tencent.com/tencentcareer/api/post/Query?timestamp={}&countryId=&cityId=2&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword=python&pageIndex=1&pageSize=10&language=zh-cn&area=cn'.format(int(time.time()*1000))
        yield scrapy.Request(url,callback=self.parse)

    def parse(self, response):
        self.p=self.p+1
        self.jsoncode=json.loads(response.text)
        # print(self.jsoncode)
        # print(response.text)

        for x in self.jsoncode['Data']['Posts']:

            detailsUrl='https://careers.tencent.com/tencentcareer/api/post/ByPostId?timestamp={}&postId={}&language=zh-cn'.format(int(time.time()*1000),x['PostId'])
            yield scrapy.Request(detailsUrl,callback=self.parse2)

        if self.p<5:
            url='https://careers.tencent.com/tencentcareer/api/post/Query?timestamp={}&countryId=&cityId=2&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword=python&pageIndex={}&pageSize=10&language=zh-cn&area=cn'.format(int(time.time()*1000),self.p)
            yield scrapy.Request(url,callback=self.parse)


        print("Postid=", self.jsoncode['Data']['Posts'][0]['PostId'])
        print('RecruitPostId= ', self.jsoncode['Data']['Posts'][0]['RecruitPostId'])
        print("RecruitPostName= ", self.jsoncode['Data']['Posts'][0]['RecruitPostName'])
        print('LocationName= ', self.jsoncode['Data']['Posts'][0]['LocationName'])
        print('BGName= ', self.jsoncode['Data']['Posts'][0]['BGName'])

    def parse2(self,response):
        detailsjson=json.loads(response.text)

        items = Tenxun1Item()
        items['CategoryName']=detailsjson['Data']['CategoryName']
        items['Responsibility']=detailsjson['Data']['Responsibility']
        items['Requirement']=detailsjson['Data']['Requirement']
        items['PostId']=detailsjson['Data']['PostId']

        items['BGName'] = detailsjson['Data']['BGName']
        items['RecruitPostName'] = detailsjson['Data']['RecruitPostName']
        items['LocationName'] = detailsjson['Data']['LocationName']

        yield items

        print("CategoryName:" + detailsjson['Data']['CategoryName'])
        print("Responsibility:" + detailsjson['Data']['Responsibility'])
        print('Requirement:' + detailsjson['Data']['Requirement'])

        # print(detailsjson)

